# 使用SimpleTagBased算法对Delicious2K数据进行推荐
# 原始数据集：https://grouplens.org/datasets/hetrec-2011/
# 数据格式：userID     bookmarkID     tagID     timestamp
import random
import math
import operator
import os
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

def to_record(org_file, record_file):

    if os.path.exists(record_file):
        records = np.load(record_file, allow_pickle=True).item()
    else:
        records = {}
        df = pd.read_csv(org_file, delimiter='\t', dtype=np.int32)
        for i in range(len(df)):
            uid = df['userID'][i]
            iid = df['bookmarkID'][i]
            tagid = df['tagID'][i]

            records.setdefault(uid, {})
            records[uid].setdefault(iid,[])
            records[uid][iid].append(tagid)
        print("数据集大小为 %d." % (len(df)))
        print("设置tag的人数 %d." % (len(records)))
        print("数据加载完成\n")
        np.save(record_file, records)
    return records

def load_dict():
    user_tags_na = np.load('./user_tags.npy', allow_pickle=True)
    user_tags = {}
    for item in user_tags_na:
        user_tags.setdefault(item[0], {}).update({item[1]: item[2]})

    tag_items_na = np.load('./tag_items.npy', allow_pickle=True)
    tag_items = {}
    for item in tag_items_na:
        tag_items.setdefault(item[0], {}).update({item[1]: item[2]})

    user_items_na = np.load('./user_items.npy', allow_pickle=True)
    user_items = {}
    for item in user_items_na:
        user_items.setdefault(item[0], {}).update({item[1]: item[2]})

    tag_users_na = np.load('./tag_users.npy', allow_pickle=True)
    tag_users = {}
    for item in tag_users_na:
        tag_users.setdefault(item[0], {}).update({item[1]: item[2]})

    return user_tags,tag_items,user_items, tag_users

def save_npy_data(filename):
    train = pd.read_csv(filename, sep='\t', dtype=np.int32)

    #train = train[['userID', 'bookmarkID', 'tagID']].astype(str)
    train = train[['userID', 'bookmarkID', 'tagID']]
    df_user_tags = train[['userID', 'tagID']].value_counts().reset_index()
    df_user_tags.rename({0: 'freq'}, axis=1, inplace=True)

    np.save('user_tags.npy', df_user_tags.values)

    df_tag_items = train[['tagID', 'bookmarkID']].value_counts().reset_index()
    df_tag_items.rename({0: 'freq'}, axis=1, inplace=True)
    np.save('tag_items.npy', df_tag_items.values)

    df_user_items = train[['userID', 'bookmarkID']].value_counts().reset_index()
    df_user_items.rename({0: 'freq'}, axis=1, inplace=True)
    np.save('user_items.npy', df_user_items.values)

    df_tag_users = train[['tagID', 'userID']].value_counts().reset_index()
    df_tag_users.rename({0: 'freq'}, axis=1, inplace=True)
    np.save('tag_users.npy', df_tag_users.values)

def load_dict_delicious(filename):
    if os.path.exists('user_tags.npy') == False:
        save_npy_data(filename)
    return load_dict()

def split_train_test(filename, train_file, test_file, ratio, seed=100):
    df = pd.read_csv(filename, sep='\t')

    train_data, test_data = train_test_split(df, test_size=ratio, random_state=seed)

    train_data.to_csv(train_file, sep='\t')
    test_data.to_csv(test_file, sep='\t')

    print("训练样本数(users)：%d，测试样本数(users)：%d" % (len(train_data), len(test_data)))

def precisionAndRecall(N):
    train_data = to_record(train_file, './train_record.npy')
    test_data = to_record(test_file, './test_record.npy')
    user_tags, tag_items, user_items, tag_users = load_dict_delicious(train_file)
    hit = 0
    h_recall = 0
    h_precision = 0
    #print("测试集数量：%d" % (len(test_data)))
    for user, items in test_data.items():
        #如果用户数据没有出现在训练集中，忽律该用户
        if user not in train_data:
            continue
        # 获取Top-N推荐列表
        rank = recommend(user, N, user_tags, tag_items, user_items, tag_users)
        for item, rui in rank:
            if item in items:
                hit = hit + 1
        h_recall = h_recall + len(items)
        h_precision = h_precision + N
    print('一共命中 %d 个, 一共推荐 %d 个, 用户设置tag总数 %d 个' %(hit, h_precision, h_recall))
    # 返回准确率 和 召回率
    return (hit / (h_precision * 1.0)), (hit / (h_recall * 1.0))

# 对用户user推荐Top-N， 返回推荐的商品和分数值
def recommend(user, N, user_tags, tag_items, user_items, tag_users):
    #user_tags, tag_items, user_items, tag_users = load_dict_delicious(train_file)
    #print("user_tags:%d, tag_items:%d, user_items:%d" % (len(user_tags),len(tag_items),len(user_items)))
    recommend_items=dict()

    wut_total = 0       #用户打过所有标签的累计
    wti_total = {}      #每一个tag被不同商品打上便签的累计次数
    wtu_total = {}      #每一个tag被不同用户打上的累计次数
    if tag_algorithm == "norm":
        for tag, wut in user_tags[user].items():
            wut_total += wut   #该用户打过的标签的累计
            wti_tag = 0
            for item, wti in tag_items[tag].items():
                wti_tag += wti #每一个tag被不同商品打上便签的累计次数
            wti_total[tag] = wti_tag
    elif tag_algorithm == "tfidf":
        for tag, wut in user_tags[user].items():
            wtu_tag = 0
            for _ in tag_users[tag].items():
                wtu_tag += 1   #每个tag被多少个不同的用户使用
            wtu_total[tag] = wtu_tag
    else:
        pass
    tagged_items = user_items[user]
    for tag, wut in user_tags[user].items():
        #print(self.user_tags[user].items())
        for item, wti in tag_items[tag].items():
            if item in tagged_items:
                continue
            #print('wut = %s, wti = %s' %(wut, wti))

            if tag_algorithm == "norm":
                if item not in recommend_items:
                    recommend_items[item] = (wut / wut_total)*  (wti / wti_total[tag])
                else:
                    recommend_items[item] = recommend_items[item] + (wut / wut_total)*  (wti / wti_total[tag])
            elif tag_algorithm == "tfidf":
                if item not in recommend_items:
                    recommend_items[item] = wut / math.log(1+wtu_total[tag]) * wti
                else:
                    recommend_items[item] = recommend_items[item] + wut / math.log(1+wtu_total[tag]) * wti
            else:
                # simple:对Item进行打分，分数为所有的（用户对某标签使用的次数 wut, 乘以 商品被打上相同标签的次数 wti）之和
                if item not in recommend_items:
                    recommend_items[item] = wut * wti
                else:
                    recommend_items[item] = recommend_items[item] + wut * wti

    return sorted(recommend_items.items(), key=operator.itemgetter(1), reverse=True)[0:N]

# 使用测试集，对推荐结果进行评估
def testRecommend():
    print("推荐结果评估")
    print("%3s %10s %10s" % ('N',"精确率",'召回率'))
    for n in [5,10,20,40,60,80,100]:
    #for n in [5]:
        precision,recall = precisionAndRecall(n)
        print("%3d %10.3f%% %10.3f%%" % (n, precision * 100, recall * 100))

data_file = './user_taggedbookmarks-timestamps.dat'
train_file = './train.csv'
test_file = './test.csv'
# tag_algorithm = 'simple'
tag_algorithm = 'tfidf'
#tag_algorithm = 'norm'

algorithm = ["simple", "norm", "tfidf"]
def main():
    #split_train_test(data_file,train_file, test_file, 0.2)
    for i in range(3):
        global tag_algorithm
        tag_algorithm = algorithm[i]
        print("%s:" % (algorithm[i]))
        testRecommend()

if __name__ == "__main__":
    main()







